NY TAXI DATA SCIENCE¶

import pandas as pd

import numpy as np

import matplotlib  

import matplotlib.pyplot as plt 

import plotly.plotly as py

from plotly.offline import download_plotlyjs, init_notebook_mode, iplot, plot

import plotly.figure_factory as ff

import plotly.graph_objs as go

from plotly import tools

from IPython.display import Image

from IPython.display import display, Math, Latex 

from IPython.core.display import HTML

import random as r

import calendar

​

​

#initiate the Plotly Notebook mode

init_notebook_mode()

​

​

df_big = pd.read_csv('../data/yellow_tripdata_2016-01.csv')

​

#optional data cleaning 

#df_big_clean=df_big.fillna(df_big.mean())#df_big.dropna(axis=1) 

#df_big_clean <- df_big[!(is.na(df$start_pc) | df$start_pc==""), ] #| is an or-operator and ! inverts. 

#Hence, the command above displays all rows, which are not b) NA or b) equal to ""

​

df_big_clean=df_big #run this line to leave data uncleaned

​

#reduce data points for testing using random sample

indexnums=r.sample(list(np.arange(1,2389990,1)), 10000)

df=df_big_clean.loc[indexnums,:]  #use to reduce data points for testing mode

#df=df_big                      # or use this line to keep whole month of data

​

#df #use this to view the dataframe

​

# what does the data cover?

df1=df_big

# Make new columns in alternate dataframe with hour of day and day of the week etc

df1['hour'] = pd.to_datetime(df1['tpep_pickup_datetime'], format='%Y-%m-%d %H:%M:%S').dt.hour

df1['day'] = pd.to_datetime(df1['tpep_pickup_datetime'], format='%Y-%m-%d %H:%M:%S').dt.dayofweek

df1['numday'] = pd.to_datetime(df1['tpep_pickup_datetime'], format='%Y-%m-%d %H:%M:%S').dt.day

df1['month'] = pd.to_datetime(df1['tpep_pickup_datetime'], format='%Y-%m-%d %H:%M:%S').dt.month

df1['year'] = pd.to_datetime(df1['tpep_pickup_datetime'], format='%Y-%m-%d %H:%M:%S').dt.year

​

print('This set contains', df_big.shape[0], 'NY Taxi  data points.')

out1=df1.year.unique()

out1.sort()

print('The data include, Years:', out1)

out2=df1.month.unique()

out2.sort()

out2=calendar.month_abbr[int(out2)]  #added int to force integer as otherwise python gave deprciation warning

print('The data include, Months:', out2)

out3=df1.numday.unique()

out3.sort()

print('The data include, days of month:', out3)

out4=df1.day.unique()

out4.sort()

weekdaystr = dict()

for i in range(0,7):

    weekdaystr[i]=calendar.day_abbr[int(out4[i])]

print('The data include, weekdays:', weekdaystr)

import numpy as np

import plotly.plotly as py

#import plotly.offline as offline

from plotly.offline import download_plotlyjs, init_notebook_mode, iplot, plot

import plotly.graph_objs as go

init_notebook_mode()

​

#extract number of people per trip

peps_per_trip_df=df.loc[:, df.columns.str.match('passenger_count')]

peps_per_trip_df.shape

#print(type(peps_per_trip_df))

peps_per_trip=df.loc[:, df.columns.str.match('passenger_count')].values

#print(type(peps_per_trip))

​

#layout=go.Layout(title="First Plot", xaxis={'title':'x1'}, yaxis={'title':'x2'})

data = [go.Histogram(x=peps_per_trip)]  #or [dataset1, darset2]

layout = go.Layout(

    title='Histogram of Passenger numbers',

    xaxis=dict(

        title='passenger number'

    ),

    yaxis=dict(

        title='Count'

    ),

    bargap=0.2,

    bargroupgap=0.1

)

​

fig = go.Figure(data=data, layout=layout)

#iplot(fig,  filename='People_per_trip_histogram') #This plots when offline; no limit; seems to be an issue with offline plot in this case, but all others in this notebook are good.  

py.iplot(fig,  filename='People_per_trip_histogram') #this plots in online mode, limit of 50/day in community a/c, need online version to work in kyso

# Distribution: Payment by type

#df=df_big  #uncomment to run on whole dataset

​

# Add histogram data

# extract fares by payment type

# 1=cc, 2=cash, 3=no charge, 4=dispute, 5=unknown, 6=voided trip

fare_paymenttype1=df.loc[df['payment_type'] == 1, 'fare_amount'].values #credit card

fare_paymenttype2=df.loc[df['payment_type'] == 2, 'fare_amount'].values #cash

#fare_paymenttype4=df.loc[df['payment_type'] == 4, 'fare_amount'].values #dispute

fare_payments=np.append(fare_paymenttype1,fare_paymenttype2)

total_paymentstype1=df.loc[df['payment_type'] == 1, 'total_amount'].values   #fare+tips+tols

total_paymentstype2=df.loc[df['payment_type'] == 2, 'total_amount'].values   #fare+tips+tols

tip_amountstype1=df.loc[df['payment_type'] == 1, 'tip_amount'].values   #fare+tips+tols

total_payments=np.append(total_paymentstype1,total_paymentstype2)

​

numberofCCpays=df.loc[df['payment_type'] == 1, 'payment_type'].sum()

numberofCashpays=df.loc[df['payment_type'] == 2, 'payment_type'].sum()/2

PcentofCCpays=np.round(numberofCCpays*100/(numberofCashpays+numberofCCpays), decimals=1)

#print(PcentofCCpays)

PcentofCashpays=np.round(numberofCashpays*100/(numberofCashpays+numberofCCpays), decimals=1)

#print(PcentofCashpays)

​

#print(type(fare_paymenttype2[1:10]))

​

# Group data together

hist_data = [fare_paymenttype1,fare_paymenttype2]

find_median1=np.median(fare_paymenttype1)

find_median2=np.median(fare_paymenttype2)

#print(find_median)

​

group_labels = ['Credit card', 'Cash']

​

​

# Create distplot with custom bin_size

fig = ff.create_distplot(hist_data, group_labels, bin_size=1.0)

fig.layout.update({'title': 'Distribution of Fares'})

fig.layout.xaxis1.update({'title': '$ amounts'})

​

​

display(Math(r'\text{Percentage of credit card payments is } %s \text{%%}' % PcentofCCpays))

display(Math(r'\text{Median credit payment is \$} %s ' % find_median1))

display(Math(r'\text{Percentage of cash payments is  } %s \text{%%}' % PcentofCashpays))

display(Math(r'\text{Median cash payment is \$} %s' % find_median2))

​

# Plot!

iplot(fig, filename='Distplot with Multiple Datasets') #offline mode

py.iplot(fig, filename='Distplot with Multiple Datasets') #online plot mode